# -*- coding: utf-8 -*-

# Define here the models for your spider middleware
#
# See documentation in:
# https://doc.scrapy.org/en/latest/topics/spider-middleware.html

from scrapy import signals
from scrapy.http.response.html import HtmlResponse
from selenium import webdriver
import time


class SeleniumMiddleWares():
    def __init__(self):
        self.driver = webdriver.Chrome(
            'E:\programlibs\chromedriver\chromedriver.exe')

    def process_request(self, request, spider):
        print('*'*100)
        print(request.url)
        print('='*100)
        self.driver.get(request.url)
        print('--'*50)
        time.sleep(2)
        try:
            while True:
                t = self.driver.find_element_by_class_name('show-more')
                t.click()
                time.sleep(0.3)
                if not t:
                    break
        except:
            print('点击发生了异常！')
        source = self.driver.page_source
        response = HtmlResponse(url=self.driver.current_url, body=source)
        return response

    def process_response(self, request, response, spider):

        return response
